"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 极致RTO-硬件故障类
Case Name   : 开启极致RTO，备机进行switchover过程中，主机注入磁盘满故障后恢复
Create At   : 2020/10/15
Owner       : @peilinqian
Description :
    1、开启极致RTO
    2、备机进行switchover
    3、注入磁盘满故障
    4、switchover失败
    5、等待10S，恢复主机故障
    6、switchover失败，重启主备；
    7、查询主备状态
    8、备机再次进行switchover
    9、还原主备，原主机进行switchover
    10、执行完成后，关闭极致RTO
    11、查询主备数据是否一致
Expect      :
    1、开启极致RTO成功
    3、备机开始进行switchover
    3、注入磁盘满故障成功
    4、switchover失败
    5、等待10S，恢复主机故障成功
    6、switchover失败，重启主备成功
    7、查询主备状态正常（如果主机状态异常，备机failover后主机进行build，代替8步骤）
    8、备机再次进行switchover成功
    9、还原主备，原主机进行switchover成功
    10、执行完成后，关闭极致RTO成功
    11、查询主备数据一致
History     :
    modified：2021-6-3 by 5328113;适配状态结果查询；
    modified：2021-4-22 by 5328113;状态结果查询方式变更，修改脚本；
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified：2021-2-5 by 5328113;guc参数还原时指定还原值
    modified:2020-12-17;tpcc执行完成后增加等待同步
    modified:2020-12-31;by @peilinqian；增加teardown主备切换异常恢复
    modified:2021-1-14;by @peilinqian；switchover过程注入主机磁盘满故障主机恢复操作
        删除不必要tpcc操作
    modified:2021-1-28 by 5328113;删除start成功断言，删除不必要tpcc
"""

import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import *
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(
            '----Opengauss_Reliability_RTO_Hardware_Disk_Case008:初始化----')

        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.tpcc_path = macro.TPCC_PATH
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            self.log.info('----1、开启极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主备是否同步----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----2、备机进行switchover----')
            switch_thread = ComThread(self.sta_1_sh.executeGsctl,
                                      args=('switchover',
                                            self.constant.SWITCH_SUCCESS_MSG,
                                            '', False))
            switch_thread.setDaemon(True)
            switch_thread.start()

            self.log.info('----3、注入磁盘满故障----')
            self.log.info("-----------获取主机磁盘信息 -----------")
            df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
            self.log.info(df_cmd)
            df_result = self.pri_root.sh(df_cmd).result()
            self.log.info(df_result)
            self.DISK_NAME = df_result.splitlines()[-1].split()[0].strip()

            self.log.info("-----------注入磁盘满故障 -----------")
            inject_cmd = f'rfile_full (diskname) values ({self.DISK_NAME})'
            inject_result = self.com.cfe_inject(self.pri_root, inject_cmd)
            self.log.info(inject_result)
            success_msg = self.constant.CFE_DISK_FULL_SUCCESS_MSG
            self.assertIn(success_msg, inject_result)

            self.log.info("-----------获取主机磁盘占用情况 -----------")
            df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
            self.log.info(df_cmd)
            df_result = self.pri_root.sh(df_cmd).result()
            self.log.info(df_result)
            self.USED_PERCENT = df_result.splitlines()[-1].split()[4].strip()
            self.assertTrue(self.USED_PERCENT == '100%')

            self.log.info('----4、switchover结果验证----')
            switch_thread.join()
            switch_result = switch_thread.get_result()
            self.log.info(switch_result)
            if switch_result:
                self.log.info('----switchover成功，refreshconf失败----')
                refresh_msg = self.sta_1_sh.exec_refresh_conf()
                self.assertFalse(refresh_msg)
            else:
                self.log.info('----switchover失败----')
                self.assertFalse(switch_result)

            self.log.info('----5、等待10S，恢复主机故障----')
            time.sleep(10)
            self.log.info("-----------清除磁盘满故障 -----------")
            clean_cmd = f'rfile_full where(diskname={self.DISK_NAME})'
            clean_result = self.com.cfe_clean(self.pri_root, clean_cmd)
            self.log.info(clean_result)
            self.assertIn(self.constant.CFE_DISK_CLEAN_SUCCESS_MSG,
                          clean_result)
            self.log.info("-----------获取主机磁盘占用情况 -----------")
            df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
            self.log.info(df_cmd)
            df_result = self.pri_root.sh(df_cmd).result()
            self.log.info(df_result)
            self.USED_PERCENT = df_result.splitlines()[-1].split()[4].strip()
            self.assertTrue(self.USED_PERCENT != '100%')

            self.log.info('----6、switchover失败，重启主备；----')
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----7、查询主备状态----')
            time.sleep(5)
            status = self.pri_sh.getDbClusterStatus('detail')
            if status.count('|') > 0:
                list_status_msg = status.splitlines()[-1].strip().split('|')
            else:
                tmp_list = status.split('[  Datanode State   ]')[-1].strip()
                list_status_msg = tmp_list.splitlines()[2::]
            self.log.info(list_status_msg)
            self.primary_status = True
            self.standby_status = True
            for item_status_msg in list_status_msg:
                if self.pri_dbuser.db_host in item_status_msg:
                    if self.constant.P_NORMAL in item_status_msg:
                        self.primary_status = True
                        self.log.info('主机状态正常')
                    else:
                        self.primary_status = False
                        self.log.info('主机状态异常')
                if self.sta1_dbuser.db_host in item_status_msg:
                    if self.constant.standby_need_repair in item_status_msg \
                            or self.constant.S_NORMAL in item_status_msg:
                        self.standby_status = True
                        self.log.info('备机状态正常')
                    else:
                        self.standby_status = False
                        self.log.info('备机状态异常')

            if not self.primary_status and self.standby_status:
                self.log.info('----8、主机异常，需要恢复主机----')
                self.log.info('----备机进行failover----')
                success_msg = self.constant.FAILOVER_SUCCESS_MSG
                failover_result = self.sta_1_sh.executeGsctl('failover',
                                                             success_msg,
                                                             '', False)
                self.assertTrue(failover_result)

                self.log.info('----进行refresh----')
                refresh_msg = self.sta_1_sh.exec_refresh_conf()
                self.assertTrue(refresh_msg)

                self.log.info('----主机重建----')
                self.pri_sh.executeGsctl('build',
                                         self.constant.REBUILD_SUCCESS_MSG)

                self.log.info('----查看现主机query，同步是否正常----')
                self.sta_1_sh.check_location_consistency('primary',
                                                         self.node_num,
                                                         self.min_time)
                self.log.info('----查询主备状态----')
                status = self.sta_1_sh.getDbClusterStatus('status')
                self.assertTrue(status)

            else:
                self.log.info('----查看主机query，同步是否正常----')
                self.pri_sh.check_location_consistency('primary',
                                                       self.node_num,
                                                       self.min_time)
                self.log.info('----查询主备状态----')
                status = self.pri_sh.getDbClusterStatus('status')
                self.assertTrue(status)

                self.log.info('----8、备机再次进行switchover----')
                success_msg = self.constant.SWITCH_SUCCESS_MSG
                switch_result = self.sta_1_sh.executeGsctl('switchover',
                                                           success_msg)
                self.assertTrue(switch_result)

                self.log.info('----进行refresh----')
                refresh_msg = self.sta_1_sh.exec_refresh_conf()
                self.assertTrue(refresh_msg)

                self.log.info('----查看现主机query，同步是否正常----')
                self.sta_1_sh.check_location_consistency('primary',
                                                         self.node_num,
                                                         self.min_time)

            self.log.info('----查询主备状态----')
            time.sleep(5)
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----9、还原主备，原主机进行switchover----')
            switch_result = self.pri_sh.executeGsctl(
                'switchover', self.constant.SWITCH_SUCCESS_MSG)
            self.assertTrue(switch_result)

            self.log.info('----进行refresh----')
            refresh_msg = self.pri_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg)

            self.pri_sh.getDbClusterStatus('status')

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

        finally:
            self.log.info('----10、执行完成后，关闭极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            # 校验备机是否完成数据同步
            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2)

            self.log.info('----11、查询主备数据是否一致----')
            nodes_tuple = (self.pri_dbuser,
                           self.sta1_dbuser,
                           self.sta2_dbuser)
            flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
            self.assertTrue(flag)

            table_dict = self.com.format_sql_result(
                self.pri_sh.executDbSql(r'\d'))
            table_name = table_dict.get('Name')
            for name in table_name:
                select_sql = f'select count(*) from {name};'
                flag = self.com.check_data_sample_by_all(select_sql,
                                                         *nodes_tuple)
                self.assertTrue(flag)

    def tearDown(self):
        df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
        self.log.info(df_cmd)
        df_result = self.pri_root.sh(df_cmd).result()
        self.log.info(df_result)
        self.USED_PERCENT = df_result.splitlines()[-1].split()[4].strip()
        if self.USED_PERCENT == '100%':
            self.log.info('----异常中断，磁盘故障未恢复处理----')
            clean_disk_cmd = f'rfile_full  where(diskname={self.DISK_NAME})'
            clean_disk_result = self.com.cfe_clean(self.pri_root,
                                                   clean_disk_cmd)
            self.log.info(clean_disk_result)

        host_name = self.pri_root.sh('hostname').result()
        if self.pri_sh.check_whether_need_switch(host_name):
            self.log.info('----恢复主备关系----')
            if self.pri_sh.check_whether_need_build():
                self.log.info('如果备机无法恢复，需要重建备机')
                self.pri_sh.executeGsctl('build',
                                         self.constant.REBUILD_SUCCESS_MSG)
            self.log.info('---进行主备切换---')
            success_msg = self.constant.SWITCHOVER_SUCCESS_MSG
            result = self.pri_sh.executeGsctl('switchover', success_msg)
            self.log.info(result)
            result = self.pri_sh.exec_refresh_conf()
            self.log.info(result)

        self.log.info('----校验刷脏页是否完成----')
        self.pri_sh.check_whether_remain_dirty_page()

        self.log.info(
            '----Opengauss_Reliability_RTO_Hardware_Disk_Case008:执行完毕----')
